package com.bbz.crawler.core.page.processor;

import java.util.List;

import org.springframework.stereotype.Component;

import com.bbz.crawler.core.util.HttpClientDownloader;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;

/**
 * 爬取知乎用户信息处理器
 * 
 * @author binbin.a.zhang
 *
 */
@Component
public class ZhiHuForPeopleProcessor implements PageProcessor {
	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(3).setSleepTime(10000).setUserAgent(
			"\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36\"")
			.setTimeOut(10000).setCharset("utf-8");

	@Override
	// process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
	public void process(Page page) {
		List<String> url = page.getHtml().xpath("//div[@class='ContentItem-image']/span/div/div/a/@href").all();
		for (int i = 0; i < url.size(); i++) {
			url.set(i, url.get(i) + "/following?page=1");
		}
		if (url == null) {
			// skip this page
			page.setSkip(true);
		} else {
			page.putField("url", url);
		}
		// URL加到抓取列表中去
		page.addTargetRequests(url);
	}

	@Override
	public Site getSite() {
		return site;
	}

	public static void main(String args[]) {
		HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
		httpClientDownloader.setProxyProvider(SimpleProxyProvider.from(new Proxy("58.253.115.112", 80)));
		Spider.create(new ZhiHuForPeopleProcessor())
				// 从https://www.zhihu.com/people/zhang-bin-51-5/following?page=1开始抓
				.addUrl("https://www.zhihu.com/people/zhang-bin-51-5/following?page=1")
				// 添加pipeline
				.addPipeline(new ConsolePipeline())
				// 开启5个线程抓取
				.thread(5).setDownloader(httpClientDownloader)
				.setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(10000000)))
				// 启动爬虫
				.run();
	}

}
